﻿'''
Created on May 12, 2011

@author: Jialai_Zhu
'''
#!/usr/bin/python 
# -*- coding: gb2312 -*- 
# 
from db import *
DB_QUERY_PHONE=mongo_query_base(db_name='zq',table_name='phone')
DB_QUERY_CERT=mongo_query_base(db_name='zq',table_name='cert')
DB_QS_S=mongo_query_base(db_name='zq',table_name='QS_S')
DB_QS_F=mongo_query_base(db_name='zq',table_name='QS_F')


def func_append_list(list,w,equal):
    for l in list:
        if equal(l,w)==True:
            return
    list.append(w)

from ptn import *
import re
class extract_fake_zq():
    def htmldecode(self,s):
        ret=reg_ptn(u'(&#x([\d|\w]{4});)+').search_all(s)
        r=[]
        for i in ret:
            u=u''
            for j in reg_ptn(u'&#x([\d|\w]{4});',1).search_all(i.extract_part,):
                print j.extract_part
                u=u+unichr(int('0x'+j.extract_part,16))  
            r.append(u) 
        return r
    post_fix_company=[u'证券责任有限公司'
                    ,u'总公司'
                    ,u'股票投资咨询有限公司'
                    ,u'股票投资公司'
                    ,u'投资理财公司'
                    ,u'投资有限公司'
                    ,u'投资咨询有限公司'
                    ,u'投资理财有限公司'
                    ,u'证券有限责任公司'
                    ,u'证券有限公司'
                    ,u'股份有限公司'
                    ,u'理财投资公司'
                    ,u'投资公司'
                    ,u'股票研究所'
                    ,u'私募投资有限责任公司'
                    ,u'私募投资有限责任公司'
                    ,u'私募投资有限责任公司'
                    ,u'证券股份 ...'
                    ,u'股份有限 ...'
                    ,u'证券股份有限公司'
                    ,u'证券有限责任 ...'
                    ,u'证券有限 ...'
                    ,u'投资 ...'
                    ,u'股票网'
                    ,u'股票研究中心'
                    ,u'证券研究中心'
                    ,u'私募软件研究所'                              
                     ]
    def init_title_ptn(self):    
#        global post_fix_company
        title_ptn=[]
        title_ptn.append(reg_ptn_template(
                           template=u'(<title>|<TITLE>|- |-|上海|北京|深圳|_)[^\u4e00-\u9fa5]*(([\u4e00-\u9fa5]+)(%(company)s))',
                           arg_list_dic={u'company':self.post_fix_company},
                           index=2))
        title_ptn.append(reg_ptn_template(
                           u'(<title>|<TITLE>|- |-|上海|北京|深圳|_|^)[^\u4e00-\u9fa5]*(([\u4e00-\u9fa5]+)(私募))',
                           2))
        get_company_name=reg_match()
        get_company_name.add_pattern_list(title_ptn)
        return get_company_name
    def init_company_html_ptn(self):    
        title_ptn=[]
        title_ptn.append(reg_ptn_template(
    #                                      证券资讯提供:上海海通证券投资理财有限公司
                           template=u'(<title>|<TITLE>)(上海|北京|深圳|)(([\u4e00-\u9fa5]+)(%(company)s))',
                           arg_list_dic={u'company':self.post_fix_company},
                           index=3))
        title_ptn.append(reg_ptn_template(
    #                                      证券资讯提供:上海海通证券投资理财有限公司
                           template=u'证券资讯提供[:|：](([\u4e00-\u9fa5]+)(%(company)s))',
                           arg_list_dic={u'company':self.post_fix_company},
                           index=1))      
        title_ptn.append(reg_ptn_template(
                           template=u'版权所有[:|：](([\u4e00-\u9fa5]+)(%(company)s))',
                           arg_list_dic={u'company':self.post_fix_company},
                           index=1))
        title_ptn.append(reg_ptn_template(
                           template=u'&copy;\s*(([\u4e00-\u9fa5]+)(%(company)s))',
                           arg_list_dic={u'company':self.post_fix_company},
                           index=1))    
        title_ptn.append(reg_ptn_template(
                           template=u'-\s*(([\u4e00-\u9fa5]+)(%(company)s))',
                           arg_list_dic={u'company':self.post_fix_company},
                           index=1))
    #    <meta name="description" content="上海广泰股票投资有限公司是您最知心的股票投资伙伴!!!">    
        get_company_name=reg_match()
        get_company_name.add_pattern_list(title_ptn)
        get_company_name.add_pattern(reg_ptn(u'证券资讯提供[:|：]([\u4e00-\u9fa5]+证券)\[',1))
        return get_company_name
    def init_shortname_ptn(self):
        short_ptn=[]
        short_ptn.append(reg_ptn_template(
                           template=u'(上海|北京|深圳|^)(([\u4e00-\u9fa5]+)%(company)s)',
                           arg_list_dic={u'company':[u'证券',u'投资',u'私募',u'股票研究所',u'股票网']},
                           index=2))
        short_ptn.append(reg_ptn_template(
                           template=u'(([\u4e00-\u9fa5]+)%(company)s)',
                           arg_list_dic={u'company':[u'证券',u'投资',u'私募',u'股票研究所',u'股票网']},
                           index=1))    
        get_company_shortname=reg_match()
        get_company_shortname.add_pattern_list(short_ptn)
        get_company_shortname.add_pattern(reg_ptn(u'大智慧'))
        return   get_company_shortname
    def init_domain_fail_ptn(self):
        m=reg_match()    
        m.add_pattern(reg_ptn(u'dot_tk_frame_content'))
        m.add_pattern(reg_ptn(u'nfdns8.cncmax.cn'))    
        m.add_pattern(reg_ptn(u'Free domain service'))
        m.add_pattern(reg_ptn(u'虚拟专用服务器'))
        m.add_pattern(reg_ptn(u'访问的域名可以转让'))
        m.add_pattern(reg_ptn(u'WingDNS'))
        m.add_pattern(reg_ptn(u'ndparking'))   
        m.add_pattern(reg_ptn(u'月付空间')) 
        m.add_pattern(reg_ptn(u'- Powered by CO.CC'))
        m.add_pattern(reg_ptn(u'购买此域名'))
        m.add_pattern(reg_ptn(u'Click here to go to'))
        
        
    #    Click here to go to
        return   m
    def init_redir_ptn(self):
        try:
            m=reg_match()    
            m.add_pattern(reg_ptn(u'location\s*=\s*[\'|\"]\s*([a-zA-z]+://(\w+(-\w+)*)(\.(\w+(-\w+)*))*(\?\S*)?)',1))    
            m.add_pattern(reg_ptn(u'td.getDate'))
            m.add_pattern(reg_ptn(u'document.write\(unescape'))
            m.add_pattern(reg_ptn(u'location=myurl'))
    #        <TITLE>&#x4ECA;&#x65E5;&#x63D0;&#x793A;        
    #        m.add_pattern(reg_ptn(u'<TITLE>.*((&#x[\d|\w]{4};)+)',1))
    #        m.add_pattern(reg_ptn(u'((&#x[\d|\w]{4};)+)',1))
        except Exception,ex:
            pass
        return   m
    def init_server_fail_ptn(self):
        m=reg_match()
        m.add_pattern(reg_ptn(u'To access IIS Help'))
        m.add_pattern(reg_ptn(u'要访问 IIS 帮助'))
        m.add_pattern(reg_ptn(u'百度--您的访问出错了'))    
        m.add_pattern(reg_ptn(u'About Apache HTTP Server'))
        m.add_pattern(reg_ptn(u'<title>找不到相关页面'))
        return   m
    def __init__(self,ptn=None):
        self.ptn_title=self.init_title_ptn()   
        self.ptn_shortname=self.init_shortname_ptn()
        self.ptn_html=self.init_company_html_ptn()
        self.ptn_server_fail=self.init_server_fail_ptn()
        self.ptn_domain_fail=self.init_domain_fail_ptn()
        self.ptn_302=self.init_redir_ptn()
        pass
    def phone_num2(self,string): 
        ret=[]
        try:
            pattern=[u'(\d{3})-(\d{8})',
                     u'18\d{9}',
                     u'13\d{9}',
                     u'15\d{9}',
                     u'18\d-\d{8}',
                     u'13\d-\d{8}',
                     u'15\d-\d{8}',]
            array_re=[]
            for  p in pattern:
                array_re.append(re.compile(p))
            for reobj in array_re:
                for match in reobj.finditer(string):
                    try:
                        s=match.start()
                        e=match.end()
                        g=match.group()
                        ret.append(g)
                    except Exception,ex:
                        pass
        except Exception,ex:
            pass       
        res=re.search(u'客服人员|证券资讯提供',string)
        if res==None:
            return [] 
        return ret 
    def phone_num(self,string):
        r=[]
        pattern=[u'(\d{3})-(\d{8})',
             u'18\d{9}',
             u'13\d{9}',
             u'15\d{9}',
             u'18\d-\d{8}',
             u'13\d-\d{8}',
             u'15\d-\d{8}',]
        for p in pattern:
            res=re.search(p, string)
            if res!=None:
                r.append(res.group())
            pass
        res=re.search(u'服务热线|客服人员|证券资讯提供',string)
        if res!=None:
            return r
        return []
    def sec_cert(self,string):
        r=[]
        try:
            if self.cert_ptn:pass
        except:
            self.cert_ptn=reg_match()
            self.cert_ptn.add_pattern(reg_ptn(ptn=u'ZX(\d+)',index=1))
            self.cert_ptn.add_pattern(reg_ptn(ptn=u'zx(\d+)',index=1))
            self.cert_ptn.add_pattern(reg_ptn(ptn=u'z(\d+)',index=1))
            self.cert_ptn.add_pattern(reg_ptn(ptn=u'Z(\d+)',index=1))
        return     self.cert_ptn.search(string) 
    def findate(self,string):
        def year(re):
            import datetime
            
            y=int(re.group(1))
            m=int(re.group(2))
            d=int(re.group(3))
            date=datetime.date(y,m,d)
            return date.strftime("%Y%m%d")
    #            return {'y':y,'m':m,'d':m}
        def date(re):
            import datetime
            i=int(re.group(1))
            today=datetime.date.today()-datetime.timedelta(days =i)
            return today.strftime("%Y%m%d")
            y=today.year
            m=today.month
            d=today.day
            return {'y':y,'m':m,'d':m}

        def hour(re):
            import datetime
            i=int(re.group(1))
            today=datetime.date.today()-datetime.timedelta(hours =i)
            return today.strftime("%Y%m%d")
            y=today.year
            m=today.month
            d=today.day
            return {'y':y,'m':m,'d':m}
        pattern={u'''(\d+)年(\d+)月(\d+)日''':year,
             u'(\d+) 天前':date,
             u'(\d+) 小时前':hour
             }        
        for p in pattern:
            res=re.search(p, string)
            if res!=None:
                return pattern[p](res)
        return ''
    def trim(self,result):
        for s in ['-',' ','_','...',' ']:
            import string
            result=string.replace(result,unicode(s),u'')
        return result
    def get_company_info(self,string,is_html):
        ret=[]
        if is_html:
            match_result=self.ptn_html.search(string)
            if match_result==None and len(string)<20:
                match_result=self.ptn_title.search(string)
        else:
            match_result=self.ptn_title.search(string)
        if match_result!=None:
            name=self.trim(match_result.extract_part)
            print name
            ret.append({'type':'qs_f','name':name})
            match_result=self.ptn_shortname.search(name)
            if match_result!=None:
                type=self.trim(match_result.extract_part)
                ret.append({'type':'qs_s','name':type})
        else:
            return None
        return ret 
    def getypte_html(self,html):
        match_result=self.__get_company_html(html)
        if match_result!=None:
            return match_result
        html_decode_list=self.htmldecode(self,html)
        for decode in html_decode_list:
            match_result=self.__get_company_html(decode)
            if match_result!=None:
                return match_result
        return None
    def __get_company_html(self,html):
        match_result=self.get_company_info(html,True)
        if match_result!=None:
            return match_result
        match_result=self.ptn_302.search(html)
        if match_result!=None:
            print match_result.match_string
            return [{'name':match_result.match_string,'type':u'302'}]
               
        match_result=self.ptn_server_fail.search(html)
        if match_result!=None:
            print match_result.match_string
            return [{'name':match_result.match_string,'type':u'server'}]
        
        match_result=self.ptn_domain_fail.search(html)
        if match_result!=None:
            print match_result.match_string
            return [{'name':match_result.match_string,'type':u'domain'}]
        return None
    def gettype(self,title,html=False):
        if html==False:
            match_result=self.get_company_info(title,False)   
        else:
            match_result=self.getypte_html(title)
        return match_result

def update_zq_company_name():
    zq_type=extract_fake_zq()
    result=[]
    from db_result import DB_QUERY_DAILY
    for item in DB_QUERY_DAILY.find({}, 0):
        try:
            title=item['title']
#            print title
            ret=zq_type.gettype(title)
            found=ret['name']
            DB_QUERY_DAILY.update_item(item, {'company':ret['type']})
            for f in result:
                if f==found:
                    found=None
                    break
            if found!=None:
                result.append(found)
        except Exception,ex:
            pass
    from db_result import gb2312file
    f=gb2312file('zqgs.txt','wb')
    for r in result:
        try:
            f.writeln(r)
        except Exception,ex:
            print ex
            pass
    pass    
def get_phone():
    zq_type=extract_fake_zq()
    cert_dic={}
    phone_dic={}
    from db_result import DB_QUERY_DAILY
    for item in DB_QUERY_DAILY.find({}, 0):
        try:
            string=item['text']
            def add2dic(dic,ret):
                if len(ret):
#                    print '\n\r'
                    for c in ret:
                        try:
                            if dic[c]:pass
                        except:
                            dic[c]=[]
                        def e(w,l):return w==l
                        func_append_list(dic[c],item['company'],e)
            ret=zq_type.phone_num(string)
            add2dic(phone_dic,ret)
            ret=zq_type.sec_cert(string)
            add2dic(cert_dic,ret)
        except Exception,ex:
            pass    
    def print_dic(dic):
        for cert in  dic:
            print cert
            for i in dic[cert]:print i
        pass  
    print_dic(phone_dic)
    print_dic(cert_dic)
    from db_result import gb2312file
    p=gb2312file('phone_company.key')
    for cert in phone_dic:
        for i in phone_dic[cert]:
            if len(i):
                key=(u'"%s" "%s"')%(cert,u'证券')
                p.writeln(key)
    p=gb2312file('cert_dic.key')
    zx=gb2312file('cert_zx.key')
    for cert in cert_dic:
        zx.writeln(cert)
        for i in cert_dic[cert]:
            if len(i):
                key=(u'"%s" "%s"')%(cert,i)
                p.writeln(key)    
def add_zq_daily(domain,keyword,time,title='',text='',option='',error='200',type='',url='',check_domain=True):
    global DB_QUERY_DAILY
    if check_domain==True:
        import db_result
        if db_result.find_zq_daily(domain)==True:
            return False
    key={}
    key['domain']=domain
    key['keyword']=keyword
    key['time']=time
    key['title']=title
    key['text']=text
    key['option']=option
    key['error']=error
    try:
        key['company']=''
        key['fullname']=''
        ZQ_TYPE=extract_fake_zq()
        n=ZQ_TYPE.gettype(title)
        key['company']=n['type']
        key['fullname']=n['name']
    except:pass

    try:
        key['detected_date']=''
        key['detected_date']=ZQ_TYPE.findate(text)
    except:        pass
    
    key['url']=url
    item=DB_QUERY_DAILY.find({'keyword':keyword,'domain':''},1)
        
    if item==None:
        DB_QUERY_DAILY.insert(key)
    else:
        DB_QUERY_DAILY.update_item(item, key)
    return True                
def main_companyname():
    zq_type=extract_fake_zq()
    for item in DB_QUERY_DAILY.find({}, 0):
        try:
            try:
                if len(item['company']):continue
            except:
                pass
            date=item['time']
            if 1:#date=='20110512':
                rets=zq_type.gettype(item['title'], False)
                for ret in rets:
                    try:
                        from db_result import DB_QUERY_DAILY
                        bcontinue=False
                        if ret['type']=='qs_s':
                            DB_QUERY_DAILY.update_item(item, {'company':ret['name']})
                            bcontinue=True
                        elif ret['type']=='qs_f':
                            DB_QUERY_DAILY.update_item(item, {'company':ret['name']})
                            bcontinue=True
                        if bcontinue:continue
                    except:
                        pass
            else:
                continue
        except Exception,ex:
            pass  
                
if __name__ == '__main__':
    s=u'&#x6D77;&#x76CA;&#x90A6;&#x6295;   &#x53F8;&#x6D77;&#x76CA;&#x90A6;&#x6295;'
    pass
    
